{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from hyperopt import Trials, STATUS_OK, tpe\n",
    "from hyperas import optim\n",
    "from hyperas.distributions import choice, uniform\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data():\n",
    "    '''\n",
    "    Data providing function:\n",
    "\n",
    "    This function is separated from model() so that hyperopt\n",
    "    won't reload data for each evaluation run.\n",
    "    '''\n",
    "    (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "    X_train = X_train.reshape(60000, 784)\n",
    "    X_test = X_test.reshape(10000, 784)\n",
    "    X_train = X_train.astype('float32')\n",
    "    X_test = X_test.astype('float32')\n",
    "    X_train /= 255\n",
    "    X_test /= 255\n",
    "    nb_classes = 10\n",
    "    Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "    Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "    return X_train, Y_train, X_test, Y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model(X_train, Y_train, X_test, Y_test):\n",
    "    '''\n",
    "    Model providing function:\n",
    "\n",
    "    Create Keras model with double curly brackets dropped-in as needed.\n",
    "    Return value has to be a valid python dictionary with two customary keys:\n",
    "        - loss: Specify a numeric evaluation metric to be minimized\n",
    "        - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
    "    The last one is optional, though recommended, namely:\n",
    "        - model: specify the model just created so that we can later use it again.\n",
    "    '''\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_shape=(784,)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout({{uniform(0, 1)}}))\n",
    "    model.add(Dense({{choice([256, 512, 1024])}}))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout({{uniform(0, 1)}}))\n",
    "    model.add(Dense(10))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    rms = RMSprop()\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])\n",
    "\n",
    "    model.fit(X_train, Y_train,\n",
    "              batch_size={{choice([64, 128])}},\n",
    "              nb_epoch=1,\n",
    "              verbose=2,\n",
    "              validation_data=(X_test, Y_test))\n",
    "    score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
    "    print('Test accuracy:', acc)\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model': model}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Imports:\n",
      "from __future__ import print_function\n",
      "\n",
      "try:\n",
      "    from hyperopt import Trials, STATUS_OK, tpe\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas import optim\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas.distributions import choice, uniform\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.models import Sequential\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.layers.core import Dense, Dropout, Activation\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.optimizers import RMSprop\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.datasets import mnist\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.utils import np_utils\n",
      "except:\n",
      "    pass\n",
      "\n",
      ">>> Hyperas search space:\n",
      "\n",
      "def get_space():\n",
      "    return {\n",
      "        'Dropout': hp.uniform('Dropout', 0, 1),\n",
      "        'Dense': hp.choice('Dense', [256, 512, 1024]),\n",
      "        'Dropout_1': hp.uniform('Dropout_1', 0, 1),\n",
      "        'batch_size': hp.choice('batch_size', [64, 128]),\n",
      "    }\n",
      "\n",
      ">>> Data\n",
      "  1: \n",
      "  2: '''\n",
      "  3: Data providing function:\n",
      "  4: \n",
      "  5: This function is separated from model() so that hyperopt\n",
      "  6: won't reload data for each evaluation run.\n",
      "  7: '''\n",
      "  8: (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
      "  9: X_train = X_train.reshape(60000, 784)\n",
      " 10: X_test = X_test.reshape(10000, 784)\n",
      " 11: X_train = X_train.astype('float32')\n",
      " 12: X_test = X_test.astype('float32')\n",
      " 13: X_train /= 255\n",
      " 14: X_test /= 255\n",
      " 15: nb_classes = 10\n",
      " 16: Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
      " 17: Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
      " 18: \n",
      " 19: \n",
      " 20: \n",
      ">>> Resulting replaced keras model:\n",
      "\n",
      "   1: def keras_fmin_fnct(space):\n",
      "   2: \n",
      "   3:     '''\n",
      "   4:     Model providing function:\n",
      "   5: \n",
      "   6:     Create Keras model with double curly brackets dropped-in as needed.\n",
      "   7:     Return value has to be a valid python dictionary with two customary keys:\n",
      "   8:         - loss: Specify a numeric evaluation metric to be minimized\n",
      "   9:         - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
      "  10:     The last one is optional, though recommended, namely:\n",
      "  11:         - model: specify the model just created so that we can later use it again.\n",
      "  12:     '''\n",
      "  13:     model = Sequential()\n",
      "  14:     model.add(Dense(512, input_shape=(784,)))\n",
      "  15:     model.add(Activation('relu'))\n",
      "  16:     model.add(Dropout(space['Dropout']))\n",
      "  17:     model.add(Dense(space['Dense']))\n",
      "  18:     model.add(Activation('relu'))\n",
      "  19:     model.add(Dropout(space['Dropout_1']))\n",
      "  20:     model.add(Dense(10))\n",
      "  21:     model.add(Activation('softmax'))\n",
      "  22: \n",
      "  23:     rms = RMSprop()\n",
      "  24:     model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])\n",
      "  25: \n",
      "  26:     model.fit(X_train, Y_train,\n",
      "  27:               batch_size=space['batch_size'],\n",
      "  28:               nb_epoch=1,\n",
      "  29:               verbose=2,\n",
      "  30:               validation_data=(X_test, Y_test))\n",
      "  31:     score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
      "  32:     print('Test accuracy:', acc)\n",
      "  33:     return {'loss': -acc, 'status': STATUS_OK, 'model': model}\n",
      "  34: \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "10s - loss: 0.3744 - acc: 0.8864 - val_loss: 0.1486 - val_acc: 0.9551\n",
      "Test accuracy: 0.9551\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "10s - loss: 0.2633 - acc: 0.9201 - val_loss: 0.1164 - val_acc: 0.9674\n",
      "Test accuracy: 0.9674\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "7s - loss: 0.5879 - acc: 0.8161 - val_loss: 0.1910 - val_acc: 0.9451\n",
      "Test accuracy: 0.9451\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "10s - loss: 0.5738 - acc: 0.8219 - val_loss: 0.1962 - val_acc: 0.9457\n",
      "Test accuracy: 0.9457\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "10s - loss: 0.4174 - acc: 0.8727 - val_loss: 0.1367 - val_acc: 0.9616\n",
      "Test accuracy: 0.9616\n"
     ]
    }
   ],
   "source": [
    "X_train, Y_train, X_test, Y_test = data()\n",
    "\n",
    "best_run, best_model = optim.minimize(model=model,\n",
    "                                      data=data,\n",
    "                                      algo=tpe.suggest,\n",
    "                                      max_evals=5,\n",
    "                                      trials=Trials(),\n",
    "                                      notebook_name='simple_notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evalutation of best performing model:\n",
      "10000/10000 [==============================] - 0s     \n",
      "[0.11977583827276249, 0.96419999999999995]\n"
     ]
    }
   ],
   "source": [
    "print(\"Evalutation of best performing model:\")\n",
    "print(best_model.evaluate(X_test, Y_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
